{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Please input your directory for the top level folder\n",
    "folder name : SUBMISSION MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_ = 'INPUT-PROJECT-DIRECTORY/submission_model/' # input only here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### setting other directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_data_dir = dir_+'2. data/'\n",
    "processed_data_dir = dir_+'2. data/processed/'\n",
    "log_dir = dir_+'4. logs/'\n",
    "model_dir = dir_+'5. models/'\n",
    "submission_dir = dir_+'6. submissions/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################################################################################\n",
    "#################### 2-2. nonrecursive model by store & cat ########################\n",
    "####################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "cvs = ['private']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "STORES = ['CA_1', 'CA_2', 'CA_3', 'CA_4', 'TX_1', 'TX_2', 'TX_3', 'WI_1', 'WI_2', 'WI_3']\n",
    "CATS = ['HOBBIES', 'HOUSEHOLD', 'FOODS']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from  datetime import datetime, timedelta\n",
    "import gc\n",
    "import numpy as np, pandas as pd\n",
    "import lightgbm as lgb\n",
    "\n",
    "import os, sys, gc, time, warnings, pickle, psutil, random\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reduce_mem_usage(df, verbose=False):\n",
    "    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n",
    "    start_mem = df.memory_usage().sum() / 1024**2    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtypes\n",
    "        if col_type in numerics:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                       df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)    \n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "FIRST_DAY = 710 \n",
    "remove_feature = ['id',\n",
    "                  'state_id',\n",
    "                  'store_id',\n",
    "#                   'item_id',\n",
    "#                   'dept_id',\n",
    "                  'cat_id',\n",
    "                  'date','wm_yr_wk','d','sales']\n",
    "\n",
    "cat_var = ['item_id', 'dept_id','store_id', 'cat_id', 'state_id'] + [\"event_name_1\", \"event_name_2\", \"event_type_1\", \"event_type_2\"]\n",
    "cat_var = list(set(cat_var) - set(remove_feature))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid2_colnm = ['sell_price', 'price_max', 'price_min', 'price_std',\n",
    "               'price_mean', 'price_norm', 'price_nunique', 'item_nunique',\n",
    "               'price_momentum', 'price_momentum_m', 'price_momentum_y']\n",
    "\n",
    "grid3_colnm = ['event_name_1', 'event_type_1', 'event_name_2',\n",
    "               'event_type_2', 'snap_CA', 'snap_TX', 'snap_WI', 'tm_d', 'tm_w', 'tm_m',\n",
    "               'tm_y', 'tm_wm', 'tm_dw', 'tm_w_end']\n",
    "\n",
    "lag_colnm = [ 'sales_lag_28', 'sales_lag_29', 'sales_lag_30',\n",
    "             'sales_lag_31', 'sales_lag_32', 'sales_lag_33', 'sales_lag_34',\n",
    "             'sales_lag_35', 'sales_lag_36', 'sales_lag_37', 'sales_lag_38',\n",
    "             'sales_lag_39', 'sales_lag_40', 'sales_lag_41', 'sales_lag_42',\n",
    "             \n",
    "             'rolling_mean_7', 'rolling_std_7', 'rolling_mean_14', 'rolling_std_14',\n",
    "             'rolling_mean_30', 'rolling_std_30', 'rolling_mean_60',\n",
    "             'rolling_std_60', 'rolling_mean_180', 'rolling_std_180']\n",
    "\n",
    "mean_enc_colnm = [\n",
    "    \n",
    "    'enc_store_id_dept_id_mean', 'enc_store_id_dept_id_std', \n",
    "    'enc_item_id_store_id_mean', 'enc_item_id_store_id_std'\n",
    "\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Make grid\n",
    "#################################################################################\n",
    "def prepare_data(store, state):\n",
    "    \n",
    "    grid_1 = pd.read_pickle(processed_data_dir+\"grid_part_1.pkl\")\n",
    "    grid_2 = pd.read_pickle(processed_data_dir+\"grid_part_2.pkl\")[grid2_colnm]\n",
    "    grid_3 = pd.read_pickle(processed_data_dir+\"grid_part_3.pkl\")[grid3_colnm]\n",
    "\n",
    "    grid_df = pd.concat([grid_1, grid_2, grid_3], axis=1)\n",
    "    del grid_1, grid_2, grid_3; gc.collect()\n",
    "    \n",
    "    grid_df = grid_df[(grid_df['store_id'] == store) & (grid_df['cat_id'] == state)]\n",
    "    grid_df = grid_df[grid_df['d'] >= FIRST_DAY]\n",
    "    \n",
    "    lag = pd.read_pickle(processed_data_dir+\"lags_df_28.pkl\")[lag_colnm]\n",
    "    \n",
    "    lag = lag[lag.index.isin(grid_df.index)]\n",
    "    \n",
    "    grid_df = pd.concat([grid_df,\n",
    "                     lag],\n",
    "                    axis=1)\n",
    "    \n",
    "    del lag; gc.collect()\n",
    "    \n",
    "\n",
    "    mean_enc = pd.read_pickle(processed_data_dir+\"mean_encoding_df.pkl\")[mean_enc_colnm]\n",
    "    mean_enc = mean_enc[mean_enc.index.isin(grid_df.index)]\n",
    "    \n",
    "    grid_df = pd.concat([grid_df,\n",
    "                         mean_enc],\n",
    "                        axis=1)    \n",
    "    del mean_enc; gc.collect()\n",
    "    \n",
    "    grid_df = reduce_mem_usage(grid_df)\n",
    "    \n",
    "    \n",
    "    \n",
    "    return grid_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation = {\n",
    "    'cv1' : [1551, 1610],\n",
    "    'cv2' : [1829,1857],\n",
    "    'cv3' : [1857, 1885],\n",
    "    'cv4' : [1885,1913],\n",
    "    'public' : [1913, 1941],\n",
    "    'private' : [1941, 1969]\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### cv1 : 2015-04-28 ~ 2015-06-26\n",
    "\n",
    "### cv2 : 2016-02-01 ~ 2016-02-28\n",
    "\n",
    "### cv3 : 2016-02-29 ~ 2016-03-27\n",
    "\n",
    "### cv4 : 2016-03-28 ~ 2016-04-24"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Model params\n",
    "#################################################################################\n",
    "lgb_params = {\n",
    "                    'boosting_type': 'gbdt',\n",
    "                    'objective': 'tweedie',\n",
    "                    'tweedie_variance_power': 1.1,\n",
    "                    'metric': 'rmse',\n",
    "                    'subsample': 0.5,\n",
    "                    'subsample_freq': 1,\n",
    "                    'learning_rate': 0.015,\n",
    "                    'num_leaves': 2**8-1,\n",
    "                    'min_data_in_leaf': 2**8-1,\n",
    "                    'feature_fraction': 0.5,\n",
    "                    'max_bin': 100,\n",
    "                    'n_estimators': 3000,\n",
    "                    'boost_from_average': False,\n",
    "                    'verbose': -1,\n",
    "                    'seed' : 1995\n",
    "                } "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pred_q(quantile):\n",
    "    print(quantile)\n",
    "\n",
    "    rmsse_bycv = dict()\n",
    "\n",
    "    for cv in cvs:\n",
    "        print('cv : day', validation[cv])\n",
    "\n",
    "        pred_list = []\n",
    "        for store in STORES:\n",
    "            for state in CATS:\n",
    "\n",
    "                print(store,state, 'start')\n",
    "                grid_df = prepare_data(store, state)\n",
    "\n",
    "                model_var = grid_df.columns[~grid_df.columns.isin(remove_feature)]\n",
    "\n",
    "                tr_mask = (grid_df['d'] <= validation[cv][0]) & (grid_df['d'] >= FIRST_DAY)\n",
    "                vl_mask = (grid_df['d'] > validation[cv][0]) & (grid_df['d'] <= validation[cv][1])\n",
    "\n",
    "                model_path = model_dir+'non_recur_model_'+store+'_'+state+'.bin'\n",
    "                m_lgb = pickle.load(open(model_path, 'rb'))\n",
    "\n",
    "\n",
    "                indice = grid_df[vl_mask].index.tolist()\n",
    "                print('starting to predict')\n",
    "                prediction = pd.DataFrame({'y_pred' : m_lgb.predict(grid_df[vl_mask][model_var], float(quantile))})\n",
    "                prediction.index = indice\n",
    "\n",
    "\n",
    "                del grid_df, m_lgb, tr_mask, vl_mask; gc.collect()\n",
    "\n",
    "                grid_1 = pd.read_pickle(processed_data_dir+\"grid_part_1.pkl\")\n",
    "                pd.concat([grid_1.iloc[indice], prediction], axis=1)\\\n",
    "                .pivot(index='id', columns='d', values='y_pred')\\\n",
    "                .reset_index()\\\n",
    "                .set_index('id')\\\n",
    "                .to_csv(log_dir+f'submission_storeandcat_{store}_{state}_{cv}.csv')\n",
    "\n",
    "\n",
    "                del grid_1; gc.collect()\n",
    "                \n",
    "    ########################### Make submissions\n",
    "    #################################################################################\n",
    "\n",
    "    os.chdir(log_dir)\n",
    "\n",
    "    pri = [a for a in os.listdir() if 'storeandcat' in a]\n",
    "\n",
    "    submission = pd.read_csv(raw_data_dir+'sample_submission.csv').set_index('id').iloc[30490:]\n",
    "    sub_id = pd.DataFrame({'id':submission.index.tolist()})\n",
    "\n",
    "    fcol = [f'F{i}' for i in range(1,29)]\n",
    "\n",
    "    sub_copy = submission.copy()\n",
    "    for file in pri:\n",
    "        temp = pd.read_csv(log_dir+file)\n",
    "        temp.columns = ['id']+fcol\n",
    "        sub_copy += sub_id.merge(temp, how='left', on='id').set_index('id').fillna(0)\n",
    "    sub_copy.columns = fcol\n",
    "    sub_copy.to_csv(submission_dir+f'before_ensemble/submission_kaggle_nonrecursive_store_cat_{quantile}.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.005\n",
      "cv : day [1941, 1969]\n",
      "CA_1 HOBBIES start\n",
      "CA_1 HOUSEHOLD start\n",
      "starting to predict\n",
      "CA_1 FOODS start\n",
      "starting to predict\n",
      "CA_2 HOBBIES start\n",
      "starting to predict\n",
      "CA_2 HOUSEHOLD start\n",
      "starting to predict\n",
      "CA_2 FOODS start\n",
      "starting to predict\n",
      "CA_3 HOBBIES start\n",
      "starting to predict\n",
      "CA_3 HOUSEHOLD start\n",
      "starting to predict\n",
      "CA_3 FOODS start\n",
      "starting to predict\n"
     ]
    }
   ],
   "source": [
    "import concurrent.futures\n",
    "for quantile in ['0.005', '0.025', '0.165', '0.250', '0.500', '0.750', '0.835', '0.975', '0.995']:\n",
    "     with concurrent.futures.ThreadPoolExecutor(\n",
    "        ) as executor:\n",
    "            executor.submit(\n",
    "                pred_q, quantile\n",
    "            )"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "conda_mxnet_p36",
   "language": "python",
   "name": "conda_mxnet_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
